{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_PATH = \"./data/\"\n",
    "TRAIN_PATH = DATA_PATH + \"train.csv\"\n",
    "TEST_PATH = DATA_PATH + \"test.csv\"\n",
    "WORD_EMBED_PATH = DATA_PATH + \"word_embed.txt\"\n",
    "CHAR_EMBED_PATH = DATA_PATH + \"char_embed.txt\"\n",
    "QUEST_PATH = DATA_PATH + \"question.csv\"\n",
    "\n",
    "train_data = pd.read_csv(TRAIN_PATH)\n",
    "test_data = pd.read_csv(TEST_PATH)\n",
    "question_data = pd.read_csv(QUEST_PATH)\n",
    "word_embedding_data = pd.read_csv(WORD_EMBED_PATH, delimiter=\" \", header=None, index_col=0)\n",
    "char_embedding_data = pd.read_csv(CHAR_EMBED_PATH, delimiter=\" \", header=None, index_col=0)\n",
    "\n",
    "question_data[\"words\"] = question_data[\"words\"].str.split(\" \")\n",
    "question_data[\"chars\"] = question_data[\"chars\"].str.split(\" \")\n",
    "\n",
    "label = train_data[\"label\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "MAX_WORD_NUMS = 10000\n",
    "\n",
    "word_tokenizer = Tokenizer(MAX_WORD_NUMS)\n",
    "word_tokenizer.fit_on_texts(question_data[\"words\"])\n",
    "\n",
    "word_embedding_data = np.concatenate(\n",
    "    (\n",
    "        np.zeros(shape=(1, word_embedding_data.shape[1]), dtype=np.float64),\n",
    "        word_embedding_data.loc[list(word_tokenizer.word_index.keys())[:MAX_WORD_NUMS]].values\n",
    "    ),\n",
    "    axis=0\n",
    ")\n",
    "word_embedding_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "WORD_SEQ_LEN = 30\n",
    "\n",
    "def gen_data(data):\n",
    "    seq_word1 = word_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q1\", right_on=\"qid\")[\"words\"])\n",
    "    seq_word2 = word_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q2\", right_on=\"qid\")[\"words\"])\n",
    "    return pad_sequences(seq_word1, maxlen=WORD_SEQ_LEN, padding=\"pre\",truncating=\"pre\"), \\\n",
    "        pad_sequences(seq_word2, maxlen=WORD_SEQ_LEN, padding=\"pre\",truncating=\"pre\")\n",
    "\n",
    "word1, word2 = gen_data(train_data)\n",
    "test_word1, test_word2 = gen_data(test_data)\n",
    "\n",
    "word1.shape, word2.shape, test_word1.shape, test_word2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.layers import LSTM, Bidirectional, TimeDistributed\n",
    "from keras.layers import Conv1D, MaxPool1D, GlobalAveragePooling1D\n",
    "from keras.layers import Input, Embedding, Dropout, BatchNormalization, Dense, Flatten, Lambda, K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# general\n",
    "NUM_EPOCHES = 50\n",
    "BATCH_SIZE = 1024\n",
    "DENSE_SIZE = 300 # 512\n",
    "DROP_RATE = 0.3\n",
    "\n",
    "# cnn\n",
    "CONV_LEN_1 = 128\n",
    "CONV_LEN_2 = 128\n",
    "CONV_LEN_3 = 128\n",
    "CONV_LEN_4 = 128\n",
    "CONV_LEN_5 = 128\n",
    "CONV_LEN_6 = 128\n",
    "CONV_LEN = CONV_LEN_1 + CONV_LEN_2 + CONV_LEN_3 + CONV_LEN_4 + CONV_LEN_5 + CONV_LEN_6\n",
    "\n",
    "# lstm\n",
    "LSTM_SIZE_1 = 256\n",
    "LSTM_SIZE_2 = 256\n",
    "DROP_RATE_LSTM = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cnn_layer_1(input1, input2, kernel_size, filters):\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    \n",
    "    conv_a = conv(input1)\n",
    "    conv_a = GlobalAveragePooling1D()(conv_a)\n",
    "    \n",
    "    conv_b = conv(input2)\n",
    "    conv_b = GlobalAveragePooling1D()(conv_b)\n",
    "    return conv_a, conv_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import train_test_split\n",
    "\n",
    "# train_word1, dev_word1, train_word2, dev_word2, train_y, dev_y = train_test_split(\n",
    "#     word1, word2, train_data[\"label\"].values,\n",
    "#     test_size=0.2\n",
    "# )\n",
    "\n",
    "# word_input1 = Input(shape=(WORD_SEQ_LEN,), dtype=\"int32\")\n",
    "# word_input2 = Input(shape=(WORD_SEQ_LEN,), dtype=\"int32\")\n",
    "\n",
    "# embedding_layer = Embedding(\n",
    "#     input_dim=word_embedding_data.shape[0],\n",
    "#     output_dim=word_embedding_data.shape[1],\n",
    "#     weights=[word_embedding_data],\n",
    "#     input_length=WORD_SEQ_LEN,\n",
    "#     trainable=False\n",
    "# )\n",
    "\n",
    "# word_vector1 = embedding_layer(word_input1)\n",
    "# word_vector2 = embedding_layer(word_input2)\n",
    "\n",
    "# lstm_layer1 = LSTM(LSTM_SIZE_1, dropout=DROP_RATE, recurrent_dropout=DROP_RATE, return_sequences=True)\n",
    "# word_first_1 = lstm_layer1(word_vector1)\n",
    "# word_first_1 = Dropout(DROP_RATE)(word_first_1)\n",
    "# word_first_2 = lstm_layer1(word_vector2)\n",
    "# word_first_2 = Dropout(DROP_RATE)(word_first_2)\n",
    "\n",
    "# lstm_layer2 = LSTM(LSTM_SIZE_2, dropout=DROP_RATE, recurrent_dropout=DROP_RATE, return_sequences=True)\n",
    "# word_second_1 = lstm_layer2(word_first_1)\n",
    "# word_second_2 = lstm_layer2(word_first_2)\n",
    "\n",
    "# conv1a, conv1b = cnn_layer_1(word_second_1, word_second_2, kernel_size=1, filters=CONV_LEN_1)\n",
    "# conv2a, conv2b = cnn_layer_1(word_second_1, word_second_2, kernel_size=2, filters=CONV_LEN_2)\n",
    "# conv3a, conv3b = cnn_layer_1(word_second_1, word_second_2, kernel_size=3, filters=CONV_LEN_3)\n",
    "# conv4a, conv4b = cnn_layer_1(word_second_1, word_second_2, kernel_size=4, filters=CONV_LEN_4)\n",
    "# conv5a, conv5b = cnn_layer_1(word_second_1, word_second_2, kernel_size=5, filters=CONV_LEN_5)\n",
    "# conv6a, conv6b = cnn_layer_1(word_second_1, word_second_2, kernel_size=6, filters=CONV_LEN_6)\n",
    "\n",
    "# merge_a = concatenate([conv1a, conv2a, conv3a, conv4a, conv5a, conv6a])\n",
    "# merge_b = concatenate([conv1b, conv2b, conv3b, conv4b, conv5b, conv6b])\n",
    "# diff = Lambda(lambda x: K.abs(x[0] - x[1]))([merge_a, merge_b])\n",
    "# mult = Lambda(lambda x: x[0] * x[1])([merge_a, merge_b])\n",
    "# merge = concatenate([diff, mult])\n",
    "\n",
    "# x = Dropout(DROP_RATE)(merge)\n",
    "# x = BatchNormalization()(x)\n",
    "\n",
    "# x = Dense(DENSE_SIZE, activation=\"relu\")(x)\n",
    "# x = Dropout(DROP_RATE)(x)\n",
    "# x = BatchNormalization()(x)\n",
    "\n",
    "# pred = Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "# model = Model(inputs=[word_input1, word_input2], outputs=pred)\n",
    "# model.compile(\n",
    "#     optimizer=\"nadam\",\n",
    "#     loss=\"binary_crossentropy\",\n",
    "#     metrics=[\"acc\"]\n",
    "# )\n",
    "\n",
    "# early_stopping = EarlyStopping(\"val_loss\", patience=10)\n",
    "# check_point = ModelCheckpoint(\n",
    "#     \"./log/%s.multi_lstm_cnn.{epoch:03d}.hdf5\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
    "#     monitor=\"val_loss\",\n",
    "#     save_best_only=True,\n",
    "#     save_weights_only=True\n",
    "# )\n",
    "\n",
    "# train_res = model.fit(\n",
    "#     x=[train_word1, train_word2],\n",
    "#     y=train_y,\n",
    "#     batch_size=BATCH_SIZE,\n",
    "#     epochs=NUM_EPOCHES,\n",
    "#     validation_data=([dev_word1, dev_word2], dev_y),\n",
    "#     shuffle=True,\n",
    "#     callbacks=[early_stopping, check_point]\n",
    "# )\n",
    "\n",
    "# pred_last = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)\n",
    "# pd.DataFrame(pred_last, columns=[\"y_pre\"]).to_csv(\n",
    "#     \"./result/%s-multilstm_cnn_pred_last.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
    "#     index=False\n",
    "# )\n",
    "\n",
    "# print(\"load model %s\" % (glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\"),))\n",
    "# model.load_weights(glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\"))\n",
    "# pred_best = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)\n",
    "# pd.DataFrame(pred_best, columns=[\"y_pre\"]).to_csv(\n",
    "#     \"./result/%s-multilstm_cnn_pred_best.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
    "#     index=False\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "best_results = []\n",
    "last_results = []\n",
    "\n",
    "for i, (train_index, dev_index) in enumerate(StratifiedKFold(n_splits=10).split(X=word1, y=label)):\n",
    "    train_word1, train_word2, train_y = word1[train_index, :], word2[train_index, :], label[train_index]\n",
    "    dev_word1, dev_word2, dev_y = word1[dev_index, :], word2[dev_index, :], label[dev_index]\n",
    "    \n",
    "    word_input1 = Input(shape=(WORD_SEQ_LEN,), dtype=\"int32\")\n",
    "    word_input2 = Input(shape=(WORD_SEQ_LEN,), dtype=\"int32\")\n",
    "\n",
    "    embedding_layer = Embedding(\n",
    "        input_dim=word_embedding_data.shape[0],\n",
    "        output_dim=word_embedding_data.shape[1],\n",
    "        weights=[word_embedding_data],\n",
    "        input_length=WORD_SEQ_LEN,\n",
    "        trainable=False\n",
    "    )\n",
    "\n",
    "    word_vector1 = embedding_layer(word_input1)\n",
    "    word_vector2 = embedding_layer(word_input2)\n",
    "\n",
    "    lstm_layer1 = LSTM(LSTM_SIZE_1, dropout=DROP_RATE, recurrent_dropout=DROP_RATE, return_sequences=True)\n",
    "    word_first_1 = lstm_layer1(word_vector1)\n",
    "    word_first_1 = Dropout(DROP_RATE)(word_first_1)\n",
    "    word_first_2 = lstm_layer1(word_vector2)\n",
    "    word_first_2 = Dropout(DROP_RATE)(word_first_2)\n",
    "\n",
    "    lstm_layer2 = LSTM(LSTM_SIZE_2, dropout=DROP_RATE, recurrent_dropout=DROP_RATE, return_sequences=True)\n",
    "    word_second_1 = lstm_layer2(word_first_1)\n",
    "    word_second_2 = lstm_layer2(word_first_2)\n",
    "\n",
    "    conv1a, conv1b = cnn_layer_1(word_second_1, word_second_2, kernel_size=1, filters=CONV_LEN_1)\n",
    "    conv2a, conv2b = cnn_layer_1(word_second_1, word_second_2, kernel_size=2, filters=CONV_LEN_2)\n",
    "    conv3a, conv3b = cnn_layer_1(word_second_1, word_second_2, kernel_size=3, filters=CONV_LEN_3)\n",
    "    conv4a, conv4b = cnn_layer_1(word_second_1, word_second_2, kernel_size=4, filters=CONV_LEN_4)\n",
    "    conv5a, conv5b = cnn_layer_1(word_second_1, word_second_2, kernel_size=5, filters=CONV_LEN_5)\n",
    "    conv6a, conv6b = cnn_layer_1(word_second_1, word_second_2, kernel_size=6, filters=CONV_LEN_6)\n",
    "\n",
    "    merge_a = concatenate([conv1a, conv2a, conv3a, conv4a, conv5a, conv6a])\n",
    "    merge_b = concatenate([conv1b, conv2b, conv3b, conv4b, conv5b, conv6b])\n",
    "    diff = Lambda(lambda x: K.abs(x[0] - x[1]))([merge_a, merge_b])\n",
    "    mult = Lambda(lambda x: x[0] * x[1])([merge_a, merge_b])\n",
    "    merge = concatenate([diff, mult])\n",
    "\n",
    "    x = Dropout(DROP_RATE)(merge)\n",
    "    x = BatchNormalization()(x)\n",
    "\n",
    "    x = Dense(DENSE_SIZE, activation=\"relu\")(x)\n",
    "    x = Dropout(DROP_RATE)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "\n",
    "    pred = Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "    model = Model(inputs=[word_input1, word_input2], outputs=pred)\n",
    "    model.compile(\n",
    "        optimizer=\"nadam\",\n",
    "        loss=\"binary_crossentropy\",\n",
    "        metrics=[\"acc\"]\n",
    "    )\n",
    "\n",
    "    early_stopping = EarlyStopping(\"val_loss\", patience=10)\n",
    "    check_point = ModelCheckpoint(\n",
    "        \"./log/%s.multi_lstm_cnn.{epoch:03d}.hdf5\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
    "        monitor=\"val_loss\",\n",
    "        save_best_only=True,\n",
    "        save_weights_only=True\n",
    "    )\n",
    "\n",
    "    train_res = model.fit(\n",
    "        x=[train_word1, train_word2],\n",
    "        y=train_y,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        epochs=NUM_EPOCHES,\n",
    "        validation_data=([dev_word1, dev_word2], dev_y),\n",
    "        shuffle=True,\n",
    "        callbacks=[early_stopping, check_point]\n",
    "    )\n",
    "\n",
    "    pred_last = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)\n",
    "    last_results.append(pd.DataFrame(pred_last, columns=[\"y_pre\"]))\n",
    "    \n",
    "\n",
    "    print(\"load model %s\" % (glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\"),))\n",
    "    model.load_weights(glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\"))\n",
    "    pred_best = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)\n",
    "    best_results.append(pd.DataFrame(pred_best, columns=[\"y_pre\"]))\n",
    "\n",
    "pd.DataFrame(pd.concat(last_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "    \"./result/%s-multilstm_cnn_pred_last.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
    "    index=False\n",
    ")\n",
    "pd.DataFrame(pd.concat(best_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "    \"./result/%s-multilstm_cnn_pred_best.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
    "    index=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train_encode = []\n",
    "# test_encode = []\n",
    "\n",
    "# for model_name in glob(\"./models/*.hdf5\"):\n",
    "#     word_input1 = Input(shape=(WORD_SEQ_LEN,), dtype=\"int32\")\n",
    "#     word_input2 = Input(shape=(WORD_SEQ_LEN,), dtype=\"int32\")\n",
    "\n",
    "#     embedding_layer = Embedding(\n",
    "#         input_dim=word_embedding_data.shape[0],\n",
    "#         output_dim=word_embedding_data.shape[1],\n",
    "#         weights=[word_embedding_data],\n",
    "#         input_length=WORD_SEQ_LEN,\n",
    "#         trainable=False\n",
    "#     )\n",
    "\n",
    "#     word_vector1 = embedding_layer(word_input1)\n",
    "#     word_vector2 = embedding_layer(word_input2)\n",
    "\n",
    "#     lstm_layer1 = LSTM(LSTM_SIZE_1, dropout=DROP_RATE, recurrent_dropout=DROP_RATE, return_sequences=True)\n",
    "#     word_first_1 = lstm_layer1(word_vector1)\n",
    "#     word_first_1 = Dropout(DROP_RATE)(word_first_1)\n",
    "#     word_first_2 = lstm_layer1(word_vector2)\n",
    "#     word_first_2 = Dropout(DROP_RATE)(word_first_2)\n",
    "\n",
    "#     lstm_layer2 = LSTM(LSTM_SIZE_2, dropout=DROP_RATE, recurrent_dropout=DROP_RATE, return_sequences=True)\n",
    "#     word_second_1 = lstm_layer2(word_first_1)\n",
    "#     word_second_2 = lstm_layer2(word_first_2)\n",
    "\n",
    "#     conv1a, conv1b = cnn_layer_1(word_second_1, word_second_2, kernel_size=1, filters=CONV_LEN_1)\n",
    "#     conv2a, conv2b = cnn_layer_1(word_second_1, word_second_2, kernel_size=2, filters=CONV_LEN_2)\n",
    "#     conv3a, conv3b = cnn_layer_1(word_second_1, word_second_2, kernel_size=3, filters=CONV_LEN_3)\n",
    "#     conv4a, conv4b = cnn_layer_1(word_second_1, word_second_2, kernel_size=4, filters=CONV_LEN_4)\n",
    "#     conv5a, conv5b = cnn_layer_1(word_second_1, word_second_2, kernel_size=5, filters=CONV_LEN_5)\n",
    "#     conv6a, conv6b = cnn_layer_1(word_second_1, word_second_2, kernel_size=6, filters=CONV_LEN_6)\n",
    "\n",
    "#     merge_a = concatenate([conv1a, conv2a, conv3a, conv4a, conv5a, conv6a])\n",
    "#     merge_b = concatenate([conv1b, conv2b, conv3b, conv4b, conv5b, conv6b])\n",
    "#     diff = Lambda(lambda x: K.abs(x[0] - x[1]))([merge_a, merge_b])\n",
    "#     mult = Lambda(lambda x: x[0] * x[1])([merge_a, merge_b])\n",
    "#     merge = concatenate([diff, mult])\n",
    "\n",
    "#     x = Dropout(DROP_RATE)(merge)\n",
    "#     x = BatchNormalization()(x)\n",
    "\n",
    "#     x = Dense(DENSE_SIZE, activation=\"relu\")(x)\n",
    "#     x = Dropout(DROP_RATE)(x)\n",
    "#     x = BatchNormalization()(x)\n",
    "\n",
    "#     pred = Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "#     model = Model(inputs=[word_input1, word_input2], outputs=pred)\n",
    "#     model.load_weights(model_name.replace(\"\\\\\", \"/\"))\n",
    "    \n",
    "#     encode_model = Model(inputs=[word_input1, word_input2], outputs=model.layers[-4].output)\n",
    "#     train_feature = encode_model.predict([word1, word2], batch_size=BATCH_SIZE)\n",
    "#     test_feature = encode_model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)\n",
    "#     train_encode.append(train_feature)\n",
    "#     test_encode.append(test_feature)\n",
    "\n",
    "# train_dense = train_encode[0].copy()\n",
    "# for t in train_encode[1:]:\n",
    "#     train_dense += t\n",
    "# train_dense = train_dense / 10\n",
    "# pd.DataFrame(train_dense).to_csv(\"train_input.csv\", index=False)\n",
    "\n",
    "# test_dense = test_encode[0].copy()\n",
    "# for t in test_encode[1:]:\n",
    "#     test_dense += t\n",
    "# test_dense = test_dense / 10\n",
    "# pd.DataFrame(test_dense).to_csv(\"test_input.csv\", index=False)\n",
    "\n",
    "# import xgboost as xgb\n",
    "# from sklearn.model_selection import train_test_split\n",
    "\n",
    "# xgb_train_x, xgb_dev_x, xgb_train_y, xgb_dev_y = train_test_split(mean_dense, label, test_size=0.2, stratify=label)\n",
    "\n",
    "# train_data = xgb.DMatrix(xgb_train_x, xgb_train_y)\n",
    "# dev_data = xgb.DMatrix(xgb_dev_x, xgb_dev_y)\n",
    "\n",
    "# params = {\n",
    "#     \"objective\": \"binary:logistic\",\n",
    "#     \"eval_metric\": \"logloss\",\n",
    "#     \"eta\": 0.01,\n",
    "#     \"max_depth\": 5,\n",
    "#     \"subsample\": 0.8,\n",
    "#     \"colsample_bytree\": 0.8,\n",
    "#     \"lambda\": 1,\n",
    "# }\n",
    "\n",
    "\n",
    "# boost = xgb.train(\n",
    "#     params=params,\n",
    "#     dtrain=train_data,\n",
    "#     num_boost_round=200,\n",
    "#     evals=[(dev_data, \"dev\")],\n",
    "#     early_stopping_rounds=10,\n",
    "# )\n",
    "\n",
    "# test_pred = boost.predict(xgb.DMatrix(test_dense))\n",
    "# pd.DataFrame(test_pred, columns=[\"y_pre\"]).to_csv(\n",
    "#     \"./result/%s-xgb_multilstm_cnn_pred_best.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
    "#     index=False\n",
    "# )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
